from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

print "Program Begins..."
#poppingPostings1IncrementalPercentSet 
#poppingPostings5IncrementalPercentSet 
#poppingPostings10IncrementalPercentSet 
#poppingPostings20IncrementalPercentSet 
#poppingPostings30IncrementalPercentSet 
#poppingPostings40IncrementalPercentSet 
#poppingPostings50IncrementalPercentSet 
#poppingPostings60IncrementalPercentSet 
#poppingPostings70IncrementalPercentSet 
#poppingPostings80IncrementalPercentSet 
#poppingPostings90IncrementalPercentSet 
#poppingPostings100IncrementalPercentSet 

# 1% 0
# 5% 1
# 10% 2
# 20% 3
# 30% 4
# 40% 5
# 50% 6
# 60% 7
# 70% 8 
# 80% 9 
# 90% 10
# 100% 11

# Updated by Wei on 2013/11/01 afternoon at school
# Q: What content are in the two lists?
# A: the actual,total / but NOT incremental posting sets
listOfSetsCompare1 = []
listOfSetsCompare2 = []

print "Part1:"
poppingPostings1IncrementalPercentList = [] #index = 0
poppingPostings5IncrementalPercentList = [] #index = 1
poppingPostings10IncrementalPercentList = [] #index = 2
poppingPostings20IncrementalPercentList = [] #index = 3
poppingPostings30IncrementalPercentList = [] #index = 4
poppingPostings40IncrementalPercentList = [] #index = 5
poppingPostings50IncrementalPercentList = [] #index = 6
poppingPostings60IncrementalPercentList = [] #index = 7
poppingPostings70IncrementalPercentList = [] #index = 8
poppingPostings80IncrementalPercentList = [] #index = 9
poppingPostings90IncrementalPercentList = [] #index = 10
poppingPostings100IncrementalPercentList = [] #index = 11
listOfThePoppingList = []
listOfThePoppingList.append(poppingPostings1IncrementalPercentList)
listOfThePoppingList.append(poppingPostings5IncrementalPercentList)
listOfThePoppingList.append(poppingPostings10IncrementalPercentList)
listOfThePoppingList.append(poppingPostings20IncrementalPercentList)
listOfThePoppingList.append(poppingPostings30IncrementalPercentList)
listOfThePoppingList.append(poppingPostings40IncrementalPercentList)
listOfThePoppingList.append(poppingPostings50IncrementalPercentList)
listOfThePoppingList.append(poppingPostings60IncrementalPercentList)
listOfThePoppingList.append(poppingPostings70IncrementalPercentList)
listOfThePoppingList.append(poppingPostings80IncrementalPercentList)
listOfThePoppingList.append(poppingPostings90IncrementalPercentList)
listOfThePoppingList.append(poppingPostings100IncrementalPercentList)

basePath = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/"
files = []
files.append("allPostingsBeingPoppedAtDifferentLevels_weight_0")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_10")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_20")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_50")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_100")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1000")
for fileName in files:
    # key: the rounding value in float format
    # value: the freq of value == the key
    rawResultHistogramDict = {}
    completePath = basePath + fileName
    print "--->",completePath
    tempFileHandler = open(completePath,"r")
    listIndexCounter = 0
    tempCounter = 0
    numOfPostingsInTotal = 0
    for line in tempFileHandler.readlines():
        if line.strip().startswith("1%") or line.strip().startswith("5%") or line.strip().startswith("10%") or line.strip().startswith("20%") or line.strip().startswith("30%") or line.strip().startswith("40%") or line.strip().startswith("50%") or line.strip().startswith("60%") or line.strip().startswith("70%") or line.strip().startswith("80%") or line.strip().startswith("90%") or line.strip().startswith("100%"):
            print "listIndexCounter:",listIndexCounter
            print "len(listOfThePoppingList[listIndexCounter]):",len(listOfThePoppingList[listIndexCounter])
            listOfSetsCompare1.append( set( listOfThePoppingList[listIndexCounter] ) )
            numOfPostingsInTotal += len(listOfThePoppingList[listIndexCounter])
            print line.strip()
            
            # add lines:
            if not line.strip().startswith("100%"):
                for element in listOfThePoppingList[listIndexCounter]:
                    listOfThePoppingList[listIndexCounter+1].append(element)
            
            listIndexCounter += 1
        elif line.strip() == "":
            pass
        else:
            listOfThePoppingList[listIndexCounter].append( line.strip() )
        tempCounter += 1
    print "tempCounter:",tempCounter
    print "numOfPostingsInTotal:",numOfPostingsInTotal
print "len(listOfSetsCompare1):",len(listOfSetsCompare1)
print len( listOfSetsCompare1[0].intersection( listOfSetsCompare1[0] ) )
print len( listOfSetsCompare1[0].union( listOfSetsCompare1[0] ) )


print "Part2:"
poppingPostings1IncrementalPercentList = [] #index = 0
poppingPostings5IncrementalPercentList = [] #index = 1
poppingPostings10IncrementalPercentList = [] #index = 2
poppingPostings20IncrementalPercentList = [] #index = 3
poppingPostings30IncrementalPercentList = [] #index = 4
poppingPostings40IncrementalPercentList = [] #index = 5
poppingPostings50IncrementalPercentList = [] #index = 6
poppingPostings60IncrementalPercentList = [] #index = 7
poppingPostings70IncrementalPercentList = [] #index = 8
poppingPostings80IncrementalPercentList = [] #index = 9
poppingPostings90IncrementalPercentList = [] #index = 10
poppingPostings100IncrementalPercentList = [] #index = 11
listOfThePoppingList = []
listOfThePoppingList.append(poppingPostings1IncrementalPercentList)
listOfThePoppingList.append(poppingPostings5IncrementalPercentList)
listOfThePoppingList.append(poppingPostings10IncrementalPercentList)
listOfThePoppingList.append(poppingPostings20IncrementalPercentList)
listOfThePoppingList.append(poppingPostings30IncrementalPercentList)
listOfThePoppingList.append(poppingPostings40IncrementalPercentList)
listOfThePoppingList.append(poppingPostings50IncrementalPercentList)
listOfThePoppingList.append(poppingPostings60IncrementalPercentList)
listOfThePoppingList.append(poppingPostings70IncrementalPercentList)
listOfThePoppingList.append(poppingPostings80IncrementalPercentList)
listOfThePoppingList.append(poppingPostings90IncrementalPercentList)
listOfThePoppingList.append(poppingPostings100IncrementalPercentList)

basePath = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/"
files = []
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_0")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_10")
files.append("allPostingsBeingPoppedAtDifferentLevels_weight_20")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_50")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_100")
#files.append("allPostingsBeingPoppedAtDifferentLevels_weight_1000")
for fileName in files:
    # key: the rounding value in float format
    # value: the freq of value == the key
    rawResultHistogramDict = {}
    completePath = basePath + fileName
    print "--->",completePath
    tempFileHandler = open(completePath,"r")
    listIndexCounter = 0
    tempCounter = 0
    numOfPostingsInTotal = 0
    for line in tempFileHandler.readlines():
        if line.strip().startswith("1%") or line.strip().startswith("5%") or line.strip().startswith("10%") or line.strip().startswith("20%") or line.strip().startswith("30%") or line.strip().startswith("40%") or line.strip().startswith("50%") or line.strip().startswith("60%") or line.strip().startswith("70%") or line.strip().startswith("80%") or line.strip().startswith("90%") or line.strip().startswith("100%"):
            print "listIndexCounter:",listIndexCounter
            print "len(listOfThePoppingList[listIndexCounter]):",len(listOfThePoppingList[listIndexCounter])
            listOfSetsCompare2.append( set( listOfThePoppingList[listIndexCounter] ) )
            numOfPostingsInTotal += len(listOfThePoppingList[listIndexCounter])
            print line.strip()
            
            # add lines:
            if not line.strip().startswith("100%"):
                for element in listOfThePoppingList[listIndexCounter]:
                    listOfThePoppingList[listIndexCounter+1].append(element)
            
            listIndexCounter += 1
        elif line.strip() == "":
            pass
        else:
            listOfThePoppingList[listIndexCounter].append( line.strip() )
        tempCounter += 1
    print "tempCounter:",tempCounter
    print "numOfPostingsInTotal:",numOfPostingsInTotal
print "len(listOfSetsCompare2):",len(listOfSetsCompare2)
print len( listOfSetsCompare2[0].intersection( listOfSetsCompare2[0] ) )
print len( listOfSetsCompare2[0].union( listOfSetsCompare2[0] ) )

print
print "Some Preliminary Results:"
print "0 & 20:"

print "1%:",len(listOfSetsCompare1[0]),len( listOfSetsCompare2[0].intersection( listOfSetsCompare1[0] ) ) / len( listOfSetsCompare2[0].union( listOfSetsCompare1[0] ) )

print "5%:",len(listOfSetsCompare1[1]),len( listOfSetsCompare2[1].intersection( listOfSetsCompare1[1] ) ) / len( listOfSetsCompare2[1].union( listOfSetsCompare1[1] ) )

print "10%:",len(listOfSetsCompare1[2]),len( listOfSetsCompare2[2].intersection( listOfSetsCompare1[2] ) ) / len( listOfSetsCompare2[2].union( listOfSetsCompare1[2] ) )

print "20%:",len(listOfSetsCompare1[3]),len( listOfSetsCompare2[3].intersection( listOfSetsCompare1[3] ) ) / len( listOfSetsCompare2[3].union( listOfSetsCompare1[3] ) )

print "30%:",len(listOfSetsCompare1[4]),len( listOfSetsCompare2[4].intersection( listOfSetsCompare1[4] ) ) / len( listOfSetsCompare2[4].union( listOfSetsCompare1[4] ) )

print "40%:",len(listOfSetsCompare1[5]),len( listOfSetsCompare2[5].intersection( listOfSetsCompare1[5] ) ) / len( listOfSetsCompare2[5].union( listOfSetsCompare1[5] ) )

print "50%:",len(listOfSetsCompare1[6]),len( listOfSetsCompare2[6].intersection( listOfSetsCompare1[6] ) ) / len( listOfSetsCompare2[6].union( listOfSetsCompare1[6] ) )

print "60%:",len(listOfSetsCompare1[7]),len( listOfSetsCompare2[7].intersection( listOfSetsCompare1[7] ) ) / len( listOfSetsCompare2[7].union( listOfSetsCompare1[7] ) )

print "70%:",len(listOfSetsCompare1[8]),len( listOfSetsCompare2[8].intersection( listOfSetsCompare1[8] ) ) / len( listOfSetsCompare2[8].union( listOfSetsCompare1[8] ) )

print "80%:",len(listOfSetsCompare1[9]),len( listOfSetsCompare2[9].intersection( listOfSetsCompare1[9] ) ) / len( listOfSetsCompare2[9].union( listOfSetsCompare1[9] ) )

print "90%:",len(listOfSetsCompare1[10]),len( listOfSetsCompare2[10].intersection( listOfSetsCompare1[10] ) ) / len( listOfSetsCompare2[10].union( listOfSetsCompare1[10] ) )

print "100%:",len(listOfSetsCompare1[11]),len( listOfSetsCompare2[11].intersection( listOfSetsCompare1[11] ) ) / len( listOfSetsCompare2[11].union( listOfSetsCompare1[11] ) )
print "Program Ends."
exit(1)

'''
# use the round function directly
print round(0.66,1)
print round(0.65,1)
print round(0.64,1)
print round(0.66,2)
exit(1)
'''

'''
# The gaps are as following:
# [0,0.1)
# [0.1,0.2)
# [0.2,0.3)
# [0.3,0.4)
# [0.4,0.5)
# [0.5,0.6)
# [0.6,0.7)
# [0.7,0.8)
# [0.8,0.9)
# [0.9,1.0]

print "Program Begins..."
basePath = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/"
files = []
files.append("debug1%")
files.append("debug5%")
files.append("debug10%")
files.append("debug20%")
files.append("debug30%")
files.append("debug40%")
files.append("debug50%")
files.append("debug60%")
files.append("debug70%")
files.append("debug80%")
files.append("debug90%")
for fileName in files:
    # key: the rounding value in float format
    # value: the freq of value == the key
    rawResultHistogramDict = {}
    completePath = basePath + fileName
    print completePath
    tempFileHandler = open(completePath,"r")
    tempCounter = 0
    for line in tempFileHandler.readlines():
        # all I want to know is the result data and that's it, so ONLY one line needs to be extracted.
        if line.startswith("----->%OfPostingsPicked:"):
            originalValueInFloatFormat = float( line.strip().split(" ")[1] )
            roundingValueInFloatFormat = round(originalValueInFloatFormat,1)
            roudningValueInStringFormat = str(roundingValueInFloatFormat)
            # print "roundingValueInFloatFormat: ",roundingValueInFloatFormat
            if roudningValueInStringFormat not in rawResultHistogramDict:
                rawResultHistogramDict[roudningValueInStringFormat] = 1
            else:
                rawResultHistogramDict[roudningValueInStringFormat] += 1
            tempCounter += 1
    # print "tempCounter:",tempCounter
    rawResultHistogramKeyList = []
    rawResultHistogramKeyList = rawResultHistogramDict.keys()
    rawResultHistogramKeyList.sort(cmp=None, key=None, reverse=False)
    for key in rawResultHistogramKeyList:
        print key,rawResultHistogramDict[key]
    print
print "Program Ends."
'''

